package com.samp.solr.file2solr;

import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import com.samp.util.MapUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

public class Statistics {

    private static final String SOLR_FIELD_CONTENT = "content";
    private static final String SOLR_FIELD_INTENTION = "intention";
    private static final String SOLR_FIELD_CONTENT_FULL_MATCH = "contentFullMatch";
    private static final String BASE_SOLR_URL = "http://iqcd-ma66:8080/solr/gcc-omcp";

    private static Integer MATCH_SUCCESS = 0;
    private static Integer MATCH_ERROR = 0;

    // distribMap 语料分布
    private static Map<String, Integer> distribMap = new HashMap<>();
    //语料对意图
    private static List<String> recordList = new ArrayList<>();

    private static Map<String, Integer> intentionsMap = new HashMap<>();

    private static void statisticsTopN(Integer tipN, boolean removeDul, Integer maxNum) {
        String sourceDir = "D:\\gcc-omcp\\new\\trans\\test\\";
        Path dir = Paths.get(sourceDir);
        //通过Files.newDirectoryStream(dir)来遍历文件夹
        try( DirectoryStream<Path> stream = Files.newDirectoryStream(dir) ){
            for(Path path : stream){
                if( !Files.isDirectory(path) ){
                    System.out.println("process........"+path.getFileName());
                    String rawPath = sourceDir+path.getFileName();
                    statisticsFile(rawPath, tipN, removeDul, maxNum);
                }
            }
            System.out.println("last result.MATCH_SUCCESS="+MATCH_SUCCESS+", MATCH_ERROR="+MATCH_ERROR);
            System.out.println("最终正确率="+(MATCH_SUCCESS.floatValue()/(MATCH_SUCCESS+MATCH_ERROR)));
        }catch(Exception e){
            e.printStackTrace();
        }
    }

    /**
     * 统计语料的分布情况。
     * @throws Exception
     */
    public static void staticWordsDistrib(String sourceDir, String outFileName) throws Exception{
        String outDir = "D:\\gcc-omcp\\new\\data\\";
        Path dir = Paths.get(sourceDir);
        //通过Files.newDirectoryStream(dir)来遍历文件夹
        try( DirectoryStream<Path> stream = Files.newDirectoryStream(dir) ){
            for(Path path : stream){
                if( !Files.isDirectory(path) ){
                    System.out.println("process........"+path.getFileName());
                    String rawPath = sourceDir+path.getFileName();
                    staticWordsDistribInFile(rawPath);
                }
            }
        }catch(Exception e){
            e.printStackTrace();
        }
//        staticWordsDistribInFile(sourceDir+"20180920.csv");
        System.out.println("last result size = "+distribMap.size());
        System.out.println("last recordList size = "+recordList.size());
        //将文件保存起来。
        long start = System.currentTimeMillis();
        BufferedWriter writer = null;
        try {
            writer = Files.newBufferedWriter(Paths.get(outDir+outFileName), StandardCharsets.UTF_8);
            for( String str: recordList ){
                writer.write(str);
                writer.newLine();
            }
            writer.flush();
        }finally{
            if( writer != null ){
                writer.close();
            }
        }
        System.out.println(" save cost : "+(System.currentTimeMillis() - start ));
        Map<String,Integer> sortedMap = MapUtils.sortedByValue(intentionsMap);
        Iterator iter = sortedMap.entrySet().iterator();
        while (iter.hasNext()) {
            Map.Entry entry = (Map.Entry) iter.next();
            Object key = entry.getKey();
            Object val = entry.getValue();
            System.out.println(key+ " founds times: "+ val);
        }
    }

    /**
     * 计算单个文件的语料分布情况。
     * @throws Exception
     */
    public static void staticWordsDistribInFile(String rawPath) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
            int i = 0 ;
            while( (line = bufferReader.readLine()) != null ){
                String transText = DataAnalyze.transferText(line);
                if( transText == null ){
                    continue;
                }
                if( distribMap.get(transText) == null ){
                    distribMap.put(transText,0);
                    recordList.add(removeLastSlot(line));
                }
                distribMap.put(transText, distribMap.get(transText)+1);
                //统计意图分布
                String[] splits = line.split("\t");
                if( splits.length == 2 ){
                    String intentin = splits[1];
                    if( intentionsMap.get(intentin) == null ){
                        intentionsMap.put(intentin, 0);
                    }
                    intentionsMap.put(intentin, intentionsMap.get(intentin)+1);
                }
                i++;
//                if( i > 20 ){
//                    break;
//                }
            }
            System.out.println("current size = "+distribMap.size());
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    private static String removeLastSlot(String text){
        String[] words = text.split("\t");
        if( words.length != 2 ){
            System.out.println("error data....."+text);
            return null;
        }
        String[] intentions = words[1].split("#");
        if( intentions.length < 4 ){
            return text;
        }
        String newIntention = intentions[0]+"#"+intentions[1]+"#"+intentions[2];
        return words[0]+"\t"+newIntention;
    }

    public static void statisticsFile(String rawPath, Integer topN, boolean removeDul, Integer maxNum) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
            SolrClient solrClient =new HttpSolrClient.Builder(BASE_SOLR_URL).build();
            int i = 0 ;
            long start = System.currentTimeMillis();
            while( (line = bufferReader.readLine()) != null ){
                if( removeDul ){
                    String transText = DataAnalyze.transferText(line);
                    if( transText == null ){
                        continue;
                    }
                    if( distribMap.get(transText) != null ){
                        continue;
                    }else {
                        distribMap.put(transText, 1);
                    }
                }
                String[] words = line.split("\t");
                if( words.length == 2 ){
                    //如果没有意图，要剔除掉。
                    if( words[1].indexOf("=") < 0 ){
                        continue;
                    }
                    List<String> topList = getTopNIntentions(solrClient, words[0], topN);
                    boolean matchIntention = false;
                    for(String topIntention : topList ){
                        if( words[1].equals(topIntention) ){
                            matchIntention = true;
//                            System.out.println("match .........yes");
//                            System.out.println("yes........."+line);
//                            System.out.println("yes........."+topIntention);
                        }else {
//                            System.out.println("match .........no");
//                            System.out.println("no........."+line);
//                            System.out.println("no........."+topIntention);
                        }
                    }
                    if( matchIntention ){
                        MATCH_SUCCESS++;
                    }else {
                        MATCH_ERROR++;
                    }
                }else {
                    System.out.println("error line:"+line);
                    continue;
                }
                if( MATCH_SUCCESS+MATCH_ERROR > maxNum ){
                    break;
                }
                i++;
                if( i % 5000 == 0  ){
                    System.out.println("cost:"+(System.currentTimeMillis()-start));
                    start = System.currentTimeMillis();
                    System.out.println("current result.MATCH_SUCCESS="+MATCH_SUCCESS+", MATCH_ERROR="+MATCH_ERROR);
                    System.out.println("当前正确率="+(MATCH_SUCCESS.floatValue())/(MATCH_SUCCESS+MATCH_ERROR));
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    private static List<String> getTopNIntentions(SolrClient solrClient, String word, Integer topN) throws Exception{
        SolrQuery query = new SolrQuery();
        query.set(CommonParams.QT, "/select");
        query.set(CommonParams.START, "0");
        query.set(CommonParams.ROWS, topN);
        String queryParam = SOLR_FIELD_CONTENT+":"+word+" OR "+ SOLR_FIELD_CONTENT_FULL_MATCH+":"+"\""+word+"\"^8";
        query.set(CommonParams.Q, queryParam);
        query.set(CommonParams.FL, SOLR_FIELD_CONTENT+","+SOLR_FIELD_INTENTION+",score");
        List<String> resultList = new ArrayList<>();
        QueryResponse response = solrClient.query(query);
        SolrDocumentList documentList = response.getResults();
        for( SolrDocument doc : documentList ) {
//            String docContent = (String) doc.getFieldValue(SOLR_FIELD_CONTENT);
            String docIntention = (String) doc.getFieldValue(SOLR_FIELD_INTENTION);
            resultList.add(docIntention);
        }
        return resultList;
    }

    public static void main(String[] args) {
//        Integer topN = Integer.parseInt(args[0]);
//        boolean removeDul = args[1].equalsIgnoreCase("true");
//        Integer maxNum = Integer.parseInt(args[2]);
//        System.out.println("start to statictisc top:"+topN);
//        System.out.println("start to statictisc removeDul:"+removeDul);
//        System.out.println("start to statictisc maxNum:"+maxNum);
//        statisticsTopN(topN, removeDul, maxNum);
        try {
//            String intention = "帮 我 接 一下\taction.name=execute#target.name=转人工#target.params1=default#target.params2=信用卡";
//            System.out.println(removeLastSlot(intention));

            String sourceDir = "D:\\gcc-omcp\\new\\trans_1_15\\";
            String outFileName = "意图分布.txt";
            staticWordsDistrib(sourceDir, outFileName);
        }catch (Exception e){
            e.printStackTrace();
        }
    }

}
